{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('tcp', 270), ('udp', 220), ('147.32.84.229', 186), ('13363', 88), ('147.32.80.9', 73), ('53', 73), ('80', 70), ('147.32.84.59', 59), ('443', 43), ('147.32.84.118', 41), ('6881', 39), ('147.32.85.25', 39), ('140.115.25.74', 30), ('147.32.84.138', 29), ('147.32.86.165', 25), ('173.242.220.227', 17), ('147.32.84.111', 17), ('188.121.222.1', 12), ('147.32.84.2', 11), ('147.32.3.51', 10), ('10010', 10), ('43087', 8), ('122.174.15.39', 7), ('12114', 7), ('147.32.87.22', 6), ('77.78.99.22', 6), ('147.32.86.24', 6), ('91.207.59.162', 6), ('203.23.120.136', 5), ('147.32.84.46', 4), ('147.32.86.166', 4), ('147.32.85.56', 4), ('147.32.86.223', 4), ('74.125.232.213', 4), ('74.125.232.220', 4), ('88.212.37.169', 3), ('94.44.60.103', 3), ('188.112.125.201', 3), ('212.201.86.130', 3), ('92.40.253.219', 3), ('147.32.80.13', 3), ('147.32.86.110', 3), ('8080', 3), ('213.155.227.215', 3), ('74.125.232.215', 3), ('161', 3), ('123', 3), ('147.32.85.18', 3), ('2.159.127.100', 2), ('85.132.162.9', 2), ('147.32.85.124', 2), ('77.78.99.21', 2), ('147.32.96.45', 2), ('147.32.84.165', 2), ('21', 2), ('89.223.179.146', 2), ('147.32.192.34', 2), ('993', 2), ('115.184.37.24', 2), ('31002', 2), ('151.41.188.39', 2), ('82.209.194.12', 2), ('194.108.204.22', 2), ('147.32.86.53', 2), ('44076', 2), ('147.32.87.220', 2), ('81.81.67.46', 2), ('62.44.1.24', 2), ('147.32.84.94', 2), ('147.32.84.164', 2), ('89.221.217.12', 2), ('1935', 2), ('22', 2), ('77.246.52.166', 2), ('194.108.204.19', 2), ('94.100.187.194', 2), ('74.125.232.216', 2), ('147.32.85.8', 2), ('109.149.84.86', 2), ('147.32.86.147', 2), ('147.32.86.111', 2), ('147.32.86.20', 2), ('88.86.102.50', 2), ('5222', 2), ('147.32.87.254', 2), ('91.123.225.57', 2), ('93.45.239.29', 1), ('62.240.166.118', 1), ('147.32.86.148', 1), ('66.235.132.232', 1), ('213.233.154.219', 1), ('95.210.161.212', 1), ('2.159.25.101', 1), ('213.233.154.226', 1), ('95.153.189.22', 1), ('77.75.73.9', 1), ('178.255.217.104', 1), ('193.6.144.62', 1), ('31.64.164.71', 1), ('66.29.191.18', 1), ('99.141.169.24', 1), ('4793', 1), ('196.201.19.50', 1), ('74.125.232.219', 1), ('195.113.232.82', 1), ('212.24.150.110', 1), ('25443', 1), ('82.39.2.249', 1), ('80.78.79.156', 1), ('49621', 1), ('82.73.244.56', 1), ('1153', 1), ('188.95.61.42', 1), ('48190', 1), ('192.221.106.126', 1), ('2774', 1), ('212.111.2.151', 1), ('147.32.86.135', 1), ('3978', 1), ('95.172.94.54', 1), ('114.78.14.160', 1), ('147.32.85.88', 1), ('56949', 1), ('195.250.146.100', 1), ('8000', 1), ('115.108.130.214', 1), ('122.176.204.167', 1), ('16041', 1), ('147.32.82.101', 1), ('147.32.86.88', 1), ('3389', 1), ('147.32.85.74', 1), ('147.32.203.229', 1), ('2971', 1), ('193.87.80.3', 1), ('51019', 1), ('95.227.158.77', 1), ('60330', 1), ('84.222.204.28', 1), ('67.218.100.83', 1), ('24.4.101.240', 1), ('147.213.68.141', 1), ('1040', 1), ('147.32.89.184', 1), ('58898', 1), ('86.161.84.198', 1), ('151.45.215.195', 1), ('1059', 1), ('59277', 1), ('192.134.164.161', 1), ('192.166.145.21', 1), ('74.125.232.214', 1), ('48102', 1), ('192.166.145.6', 1), ('49198', 1), ('85.237.224.48', 1), ('89.135.152.106', 1), ('1126', 1), ('211.101.37.103', 1), ('195.72.134.115', 1), ('195.130.82.45', 1), ('116.50.166.74', 1), ('46.5.184.57', 1), ('36165', 1), ('217.150.190.17', 1), ('74.125.108.208', 1), ('2768', 1), ('41.130.235.95', 1), ('212.239.26.199', 1), ('51572', 1), ('58.182.2.6', 1), ('147.32.87.30', 1), ('46.158.48.2', 1), ('178.200.203.145', 1), ('7300', 1), ('81.57.190.106', 1), ('6911', 1), ('77.127.196.107', 1), ('147.32.84.73', 1), ('113.161.71.62', 1), ('38217', 1), ('115.173.243.205', 1), ('194.161.219.176', 1), ('51003', 1), ('31.162.200.244', 1), ('174.98.117.98', 1), ('77.126.195.83', 1), ('23065', 1), ('85.207.55.78', 1), ('130.192.181.149', 1), ('68.51.197.191', 1), ('21348', 1), ('114.26.174.109', 1), ('31056', 1), ('77.85.143.253', 1), ('95.149.168.251', 1), ('81.20.48.51', 1), ('39836', 1), ('94.179.103.184', 1), ('84.12.54.36', 1), ('203.125.50.50', 1), ('2.32.222.83', 1), ('195.24.232.164', 1), ('34588', 1), ('196.44.240.61', 1), ('49.240.140.102', 1), ('80.239.149.26', 1), ('201.209.15.135', 1), ('94.126.186.67', 1), ('86.100.133.199', 1), ('58.8.226.198', 1), ('223.205.214.227', 1), ('98.244.73.136', 1), ('147.32.97.253', 1), ('147.32.84.8', 1), ('113.65.40.102', 1), ('195.250.146.6', 1), ('49340', 1), ('53538', 1), ('78.128.194.184', 1), ('147.32.84.68', 1), ('52948', 1), ('78.80.14.227', 1), ('147.32.84.3', 1), ('41343', 1), ('109.164.3.183', 1), ('85.70.149.37', 1), ('3360', 1), ('141.156.150.237', 1), ('82.199.111.22', 1), ('46.191.167.153', 1), ('195.239.66.250', 1), ('50973', 1), ('147.32.85.34', 1), ('35149', 1), ('67.160.121.194', 1), ('126.170.234.66', 1), ('95.52.90.246', 1), ('53185', 1), ('78.84.137.106', 1), ('30657', 1), ('116.255.12.184', 1), ('55584', 1), ('24.247.253.8', 1), ('34092', 1), ('83.11.238.15', 1), ('147.32.86.239', 1), ('62.24.66.210', 1), ('178.90.23.47', 1), ('147.32.85.89', 1), ('147.32.85.26', 1), ('209.85.148.147', 1), ('109.175.50.70', 1), ('1194', 1), ('117.194.42.208', 1), ('33482', 1), ('114.138.217.214', 1), ('9146', 1), ('178.253.242.3', 1), ('178.2.216.156', 1), ('147.32.84.21', 1), ('44146', 1), ('78.90.84.134', 1), ('147.230.32.193', 1), ('194.228.143.168', 1), ('16516', 1), ('90.183.39.86', 1), ('85.13.72.246', 1), ('147.32.87.2', 1), ('76.13.114.90', 1), ('81.90.173.232', 1), ('88.177.153.150', 1), ('23739', 1), ('114.45.250.190', 1), ('1942', 1), ('147.32.87.48', 1), ('147.32.86.252', 1), ('82.177.229.240', 1), ('41.241.93.111', 1), ('76.164.192.82', 1), ('147.32.86.117', 1), ('16476', 1), ('130.236.37.3', 1), ('57989', 1), ('90.183.38.8', 1), ('49463', 1), ('218.209.43.8', 1), ('87.244.151.27', 1), ('31518', 1), ('89.178.50.15', 1), ('1273', 1), ('94.179.93.205', 1), ('3605', 1), ('74.125.232.202', 1), ('94.74.233.93', 1), ('113.105.171.56', 1), ('147.32.86.66', 1), ('40565', 1), ('192.100.130.7', 1), ('212.235.185.142', 1), ('20992', 1), ('21835', 1), ('147.32.84.184', 1), ('199.59.149.200', 1), ('188.175.127.220', 1), ('203.99.179.112', 1), ('74.215.40.251', 1), ('159.148.227.125', 1), ('49230', 1), ('60.250.102.130', 1), ('94.198.111.82', 1), ('193.142.0.1', 1), ('193.165.4.137', 1), ('183.83.130.244', 1), ('85.117.224.50', 1), ('67.195.115.177', 1), ('147.32.87.27', 1), ('147.32.85.5', 1), ('52316', 1), ('83.228.35.75', 1), ('78.141.177.63', 1), ('194.129.65.79', 1), ('109.153.100.95', 1), ('84.217.37.3', 1), ('55840', 1), ('205.188.10.189', 1), ('95.32.5.235', 1), ('109.11.239.168', 1), ('201.79.47.179', 1), ('54239', 1), ('182.170.250.154', 1), ('55316', 1), ('79.50.93.42', 1), ('45974', 1), ('222.145.103.227', 1), ('69.238.171.67', 1), ('77.120.178.242', 1), ('15451', 1), ('124.102.16.132', 1), ('195.18.192.176', 1), ('118.165.82.114', 1), ('42796', 1), ('85.14.18.164', 1), ('32410', 1), ('192.168.1.99', 1), ('110.1.150.128', 1), ('188.138.84.239', 1), ('178.126.213.97', 1), ('62.118.200.4', 1), ('76.91.139.18', 1), ('5318', 1), ('91.148.14.80', 1), ('88.1.124.189', 1), ('188.244.37.212', 1), ('151.50.42.148', 1), ('90.178.130.71', 1), ('219.95.99.182', 1), ('81.145.244.43', 1), ('86.63.194.225', 1), ('129.6.15.28', 1), ('119.82.68.244', 1), ('3689', 1), ('64.228.86.17', 1), ('59156', 1), ('68.227.9.121', 1), ('38769', 1), ('94.155.203.122', 1), ('41688', 1), ('77.37.194.251', 1), ('10733', 1), ('118.233.160.13', 1), ('29930', 1), ('98.166.254.26', 1), ('16775', 1), ('78.141.181.221', 1), ('34049', 1), ('61.219.106.199', 1), ('22015', 1), ('79.180.232.121', 1), ('33791', 1), ('121.218.171.254', 1), ('59229', 1), ('119.194.132.184', 1), ('94.45.56.221', 1), ('14711', 1), ('88.160.241.60', 1), ('13631', 1), ('85.122.145.220', 1), ('63231', 1), ('24.218.41.238', 1), ('42643', 1), ('82.243.178.156', 1), ('24211', 1), ('96.20.140.58', 1), ('55397', 1), ('60.250.85.109', 1), ('37514', 1), ('114.40.141.216', 1), ('18082', 1), ('60.248.145.115', 1), ('8206', 1), ('68.144.201.22', 1), ('58975', 1), ('70.64.246.26', 1), ('16832', 1), ('84.229.5.224', 1), ('8824', 1), ('46.127.53.27', 1), ('44593', 1), ('92.247.216.178', 1), ('52613', 1), ('88.193.88.76', 1), ('58676', 1), ('84.38.184.4', 1), ('49341', 1), ('147.32.86.181', 1), ('147.32.3.12', 1), ('4628', 1), ('78.231.189.116', 1), ('63550', 1), ('91.207.59.161', 1), ('2641', 1), ('62.245.98.34', 1), ('14065', 1), ('82.135.148.158', 1), ('147.32.86.6', 1), ('208.46.117.171', 1), ('3478', 1), ('149.5.45.8', 1), ('43017', 1), ('84.42.148.179', 1), ('49653', 1), ('85.193.5.150', 1), ('195.110.40.7', 1), ('174.48.220.201', 1), ('178.4.36.49', 1), ('201.231.168.71', 1), ('147.229.74.152', 1), ('43117', 1), ('194.242.41.77', 1), ('84.112.212.191', 1), ('13723', 1), ('77.72.244.45', 1), ('26388', 1), ('213.226.244.55', 1), ('2739', 1), ('220.130.40.204', 1), ('12748', 1), ('122.116.188.88', 1), ('18510', 1), ('128.32.224.18', 1), ('37200', 1), ('76.27.46.107', 1), ('55557', 1), ('69.204.160.28', 1), ('23055', 1), ('87.0.21.115', 1), ('130.75.243.145', 1), ('3312', 1), ('205.188.8.254', 1), ('1071', 1), ('125.66.203.116', 1), ('4669', 1), ('2755', 1), ('2756', 1), ('122.174.115.150', 1), ('2164', 1), ('147.32.86.77', 1), ('68.37.66.192', 1), ('13873', 1), ('41.132.73.182', 1), ('23018', 1), ('94.127.76.194', 1), ('50188', 1), ('2762', 1), ('2760', 1), ('2759', 1), ('63.135.80.58', 1), ('66.150.244.242', 1), ('137.254.16.69', 1), ('147.32.85.103', 1), ('49311', 1), ('213.226.63.145', 1), ('46.10.96.224', 1), ('3412', 1), ('218.43.165.32', 1), ('192.84.221.33', 1), ('55000', 1), ('110.134.149.49', 1), ('40261', 1), ('147.32.85.60', 1), ('77.78.110.71', 1), ('113.234.168.192', 1), ('195.200.251.89', 1)]\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import collections\n",
    "import tensorflow as tf\n",
    "import gensim\n",
    "\n",
    "def createNeighbor(df):\n",
    "    data = []\n",
    "    for each_line in df.iterrows():\n",
    "        for header in [\"Proto\",\"SrcAddr\",\"Dport\"]:\n",
    "            if header == \"SrcAddr\":\n",
    "                data.append([each_line[1][header],each_line[1][\"DstAddr\"]])\n",
    "                data.append([each_line[1][header],each_line[1][\"Proto\"]])\n",
    "                data.append([each_line[1][header],str(each_line[1][\"Dport\"])])\n",
    "            else:\n",
    "                data.append([str(each_line[1][header]),each_line[1][\"DstAddr\"]])\n",
    "\n",
    "    return data\n",
    "\n",
    "df = pd.read_csv(\"corpus_example.csv\")\n",
    "\n",
    "word2int={}\n",
    "sentences = []\n",
    "wordlist = []\n",
    "for each_line in df.iterrows():\n",
    "    sentences.append(each_line[1].tolist())\n",
    "    for header in [\"Proto\",\"SrcAddr\",\"DstAddr\",\"Dport\"]:\n",
    "        wordlist.append(str(each_line[1][header]))\n",
    "\n",
    "myCounter = collections.Counter(wordlist)\n",
    "print(myCounter.most_common())\n",
    "wordlist=set(wordlist)\n",
    "\n",
    "neighbor = createNeighbor(df)\n",
    "hi = pd.DataFrame(neighbor, columns = ['input', 'label'])\n",
    "\n",
    "for i,word in enumerate(wordlist):\n",
    "    word2int[word] = i\n",
    "\n",
    "#for each in word2int.keys():\n",
    "#    print(each+\":\"+str(word2int[each]))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Proto</th>\n",
       "      <th>SrcAddr</th>\n",
       "      <th>DstAddr</th>\n",
       "      <th>Dport</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tcp</td>\n",
       "      <td>93.45.239.29</td>\n",
       "      <td>147.32.84.118</td>\n",
       "      <td>6881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tcp</td>\n",
       "      <td>62.240.166.118</td>\n",
       "      <td>147.32.84.229</td>\n",
       "      <td>13363</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>tcp</td>\n",
       "      <td>147.32.86.148</td>\n",
       "      <td>66.235.132.232</td>\n",
       "      <td>80</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tcp</td>\n",
       "      <td>147.32.3.51</td>\n",
       "      <td>147.32.84.46</td>\n",
       "      <td>10010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>tcp</td>\n",
       "      <td>88.212.37.169</td>\n",
       "      <td>147.32.84.118</td>\n",
       "      <td>6881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Proto         SrcAddr         DstAddr  Dport\n",
       "0   tcp    93.45.239.29   147.32.84.118   6881\n",
       "1   tcp  62.240.166.118   147.32.84.229  13363\n",
       "2   tcp   147.32.86.148  66.235.132.232     80\n",
       "3   tcp     147.32.3.51    147.32.84.46  10010\n",
       "4   tcp   88.212.37.169   147.32.84.118   6881"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>tcp</td>\n",
       "      <td>147.32.84.118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>93.45.239.29</td>\n",
       "      <td>147.32.84.118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>93.45.239.29</td>\n",
       "      <td>tcp</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93.45.239.29</td>\n",
       "      <td>6881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6881</td>\n",
       "      <td>147.32.84.118</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          input          label\n",
       "0           tcp  147.32.84.118\n",
       "1  93.45.239.29  147.32.84.118\n",
       "2  93.45.239.29            tcp\n",
       "3  93.45.239.29           6881\n",
       "4          6881  147.32.84.118"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hi.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ONE_HOT_DIM = len(wordlist)\n",
    "\n",
    "# function to convert numbers to one hot vectors\n",
    "def to_one_hot_encoding(data_point_index):\n",
    "    one_hot_encoding = np.zeros(ONE_HOT_DIM)\n",
    "    one_hot_encoding[data_point_index] = 1\n",
    "    return one_hot_encoding\n",
    "\n",
    "X = [] # input word\n",
    "Y = [] # target word\n",
    "\n",
    "for x, y in zip(hi['input'], hi['label']):\n",
    "    X.append(to_one_hot_encoding(word2int[ x ]))\n",
    "    Y.append(to_one_hot_encoding(word2int[ y ]))\n",
    "\n",
    "# convert them to numpy arrays\n",
    "X_train = np.asarray(X)\n",
    "Y_train = np.asarray(Y)\n",
    "\n",
    "# making placeholders for X_train and Y_train\n",
    "x = tf.placeholder(tf.float32, shape=(None, ONE_HOT_DIM))\n",
    "y_label = tf.placeholder(tf.float32, shape=(None, ONE_HOT_DIM))\n",
    "\n",
    "# word embedding will be 2 dimension for 2d visualization\n",
    "EMBEDDING_DIM = 100\n",
    "\n",
    "# hidden layer: which represents word vector eventually\n",
    "W1 = tf.Variable(tf.random_normal([ONE_HOT_DIM, EMBEDDING_DIM]))\n",
    "b1 = tf.Variable(tf.random_normal([1])) #bias\n",
    "hidden_layer = tf.add(tf.matmul(x,W1), b1)\n",
    "\n",
    "# output layer\n",
    "W2 = tf.Variable(tf.random_normal([EMBEDDING_DIM, ONE_HOT_DIM]))\n",
    "b2 = tf.Variable(tf.random_normal([1]))\n",
    "prediction = tf.nn.softmax(tf.add( tf.matmul(hidden_layer, W2), b2))\n",
    "\n",
    "# loss function: cross entropy\n",
    "loss = tf.reduce_mean(-tf.reduce_sum(y_label * tf.log(prediction), axis=[1]))\n",
    "\n",
    "# training operation\n",
    "train_op = tf.train.GradientDescentOptimizer(0.05).minimize(loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iteration 0 loss is :  nan\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-66-904dfe59ce94>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0;31m# input is X_train which is one hot encoded word\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;31m# label is Y_train which is one hot encoded neighbor word\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_label\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m10\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'iteration '\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m' loss is : '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m{\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mX_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_label\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    893\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    894\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 895\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    896\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    897\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1122\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1123\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1124\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1125\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1126\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1319\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1320\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1321\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1322\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1323\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1325\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1328\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1304\u001b[0m           return tf_session.TF_Run(session, options,\n\u001b[1;32m   1305\u001b[0m                                    \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1306\u001b[0;31m                                    status, run_metadata)\n\u001b[0m\u001b[1;32m   1307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1308\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "sess = tf.Session()\n",
    "init = tf.global_variables_initializer()\n",
    "sess.run(init) \n",
    "\n",
    "iteration = 2000\n",
    "for i in range(iteration):\n",
    "    # input is X_train which is one hot encoded word\n",
    "    # label is Y_train which is one hot encoded neighbor word\n",
    "    sess.run(train_op, feed_dict={x: X_train, y_label: Y_train})\n",
    "    if i % 10 == 0:\n",
    "        print('iteration '+str(i)+' loss is : ', sess.run(loss, feed_dict={x: X_train, y_label: Y_train}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectors = sess.run(W1 + b1)\n",
    "print(vectors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "w2v_df = pd.DataFrame(vectors, columns = [\"x\"+str(i) for i in range(1,101) ])\n",
    "w2v_df['word'] = wordlist\n",
    "head_list = [\"x\"+str(i) for i in range(1,101)]\n",
    "w2v_df = w2v_df[['word']+head_list]\n",
    "w2v_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\"Creates and TSNE model and plots it\"\n",
    "labels = []\n",
    "tokens = []\n",
    "\n",
    "    \n",
    "tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n",
    "new_values = tsne_model.fit_transform(vectors)\n",
    "\n",
    "x = []\n",
    "y = []\n",
    "for value in new_values:\n",
    "    x.append(value[0])\n",
    "    y.append(value[1])\n",
    "        \n",
    "plt.figure(figsize=(16, 16)) \n",
    "for i in range(len(x)):\n",
    "    plt.scatter(x[i],y[i],marker='o',c='blue')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.cluster import KMeans\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "k_cluster = pd.DataFrame(new_value,columns=['x', 'y'])\n",
    "\n",
    "# visualize data point\n",
    "sns.lmplot('x', 'y', data=k_cluster, fit_reg=False, scatter_kws={\"s\": 200}) # x-axis, y-axis, data, no line, marker size\n",
    "\n",
    "# title\n",
    "plt.title('kmean plot')\n",
    "\n",
    "# x-axis label\n",
    "plt.xlabel('x')\n",
    "# y-axis label\n",
    "plt.ylabel('y')\n",
    "# convert dataframe to numpy array\n",
    "data_points = k_cluster.values\n",
    "kmeans = KMeans(n_clusters=3).fit(data_points)\n",
    "kmeans.labels_\n",
    "kmeans.cluster_centers_\n",
    "k_cluster['cluster_id'] = kmeans.labels_\n",
    "k_cluster.head(12)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sns.lmplot('x', 'y', data=k_cluster, fit_reg=False,  # x-axis, y-axis, data, no line\n",
    "           scatter_kws={\"s\": 150}, # marker size\n",
    "           hue=\"cluster_id\") # color\n",
    "\n",
    "# title\n",
    "plt.title('after kmean clustering')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
